package engine

import (
	"log"
	"wjerp.com/hello/crawler/fetcher"
)

type SimpleEngine struct {}

func (e SimpleEngine) Run(seeds ...Request){
	var requests []Request
	for _,req:= range seeds{
		requests = append(requests,req)
	}

	for len(requests) > 0{
		req := requests[0]
		requests = requests[1:]
		parseResult, err := worker(req)
		if err != nil {
			continue
		}
		requests = append(requests, parseResult.Requests...)
		for _,item := range parseResult.Items{
			log.Printf("Got item %v",item)

		}
	}
}

func worker(r Request)(ParseResult,error){
	log.Printf("Fetcher Url :%s\n",r.Url)
	body , err := fetcher.Fetch(r.Url)
	if err != nil {
		log.Printf("Fetcher error fetcher url %s:%v\n",r.Url, err)
		return ParseResult{},err
	}
	return r.ParseFunc(body),nil
}
